// OpentTxl-C Version 11 compiled app store/load facility
// J.R. Cordy, Jan 2023

// Copyright 2023, James R. Cordy and others

// Permission is hereby granted, free of charge, to any person obtaining a copy of this software 
// and associated documentation files (the “Software”), to deal in the Software without restriction, 
// including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, 
// and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, 
// subject to the following conditions:

// The above copyright notice and this permission notice shall be included in all copies 
// or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, 
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE 
// AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, 
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

// TXL Load/Store/Standalone Facility
// Provides storing and loading of compiled TXL program as bytecode
// Used by TXL app compiler (txlc) to compile C-based standalone TXL apps

// Compressed save and restore for compiled TXL programs as structured bytecode.
//
// The optimizations used in this module depend implicitly on several things.
//
//      (i) The implementation of the 'trees' and 'kids' used to represent
//              trees must be in arrays rather than the more natural collections.
//
//      (ii) The parser (parse.ch), grammar tree builder (compdef.ch)
//              and rule table builder (comprul.ch) must not try to 
//              'share' any trees or kids with the bootstrap or parse of the 
//              TXL program (i.e., they must explicitly create new trees
//              and kids for everything they build).
//              
//      (iii) All pre-initialized trees and kids, such as 'empty', are always
//              allocated first and lie completely within the first 'reservedTrees', 'reservedKids' 
//              elements of the 'kids' and 'trees' arrays.
//              

// Modification Log

// v11.0 Initial revision, adapted from OpenTxl 11.0

// v11.3 Added multiple skipping criteria

#define LOADSTORE

// I/O, strings, memory allocation
#include "support.h"

// Global modules
#include "locale.h"
#include "limits.h"
#include "options.h"
#include "tokens.h"
#include "trees.h"
#include "errors.h"
#include "shared.h"
#include "charset.h"
#include "idents.h"
#include "symbols.h"

// Compiler modules
#include "rules.h"
#include "scan.h"

// Check interface consistency
#include "loadstor.h"

// Target grammar tree
extern treePT inputGrammarTreeTP;

// Magic number used to detect synchronization / version errors in stored bytecode
static int loadstore_MAGNUM;  // 0721696969 (pizza in Karlsruhe)

static void loadstore_CompressKids (void) {
    // This procedure compresses out tree kids used in the bootstrap and txl source analysis stages 
    // to compact kids used in the user's TXL program to the beginning of the kids array.
    // This maximizes the free kid space available for transformation when the compiled app is run.

    // We can't compress if we've already run out of kid space
    assert (tree_allocationStrategy == simple);
    
    // Step 1. Compress Kids
    //
    // tree kid pointers appear only in one place:
    //
    //  tree.trees ->kidsKP

    if (tree_firstUserKid <= reservedKids) {
        return;
    }

    // kid compression shift - we can eliminate any kids used before the user's TXL program
    const int kidShift = reservedKids - tree_firstUserKid;

    // (i) shift all kid pointers in tree.trees ->kidsKP
    for (int n = tree_firstUserTree; n <= tree_treeCount; n++) {
        switch (tree_trees[n].kind) {
            case treeKind_order: case treeKind_choose: case treeKind_repeat: case treeKind_leftchoose: 
            case treeKind_generaterepeat: case treeKind_list: case treeKind_generatelist: case treeKind_lookahead: 
            case treeKind_push: case treeKind_pop: case treeKind_ruleCall: case treeKind_expression: case treeKind_lastExpression :
                {
                    if ((tree_trees[n].kidsKP) >= tree_firstUserKid) {
                        tree_trees[n].kidsKP += kidShift;
                    }
                }
                break;
            default :
                break;
        }
    }

    // (ii) move the actual shifted kids down to the beginning of the array
    for (int c = tree_firstUserKid; c <= tree_kidCount; c++) {
        tree_kids[c + kidShift] = tree_kids[c];
    }

    // document what we've done
    tree_firstUserKid += kidShift;
    tree_kidCount += kidShift;
}

static void loadstore_CompressTrees (void) {
    // This procedure compresses out tree nodes used in the bootstrap and txl source analysis stages 
    // to compact tree nodes used in the user's TXL program to the beginning of the trees array.
    // This maximizes the free tree space available for transformation when the compiled app is run.

    // We can't compress if we've already run out of tree space
    assert (tree_allocationStrategy == simple);

    // Step 2. Compress Trees
    //
    // tree pointers appear only in four places:
    //
    //  (i) tree.kids (*)
    //  (ii) rules ->patternTP, rules ->replacementTP, 
    //       rules ->prePattern.partsBase ->patternTP,
    //       rules ->prePattern.partsBase ->replacementTP,
    //       rules ->postPattern.partsBase ->patternTP,
    //       rules ->postPattern.partsBase ->replacementTP
    //  (iii) inputGrammarTreeTP
    //  (iv) symbol.symbols

    if (tree_firstUserTree <= reservedTrees) {
        return;
    }

    // tree compression shift - we can eliminate any tree nodes used before the user's TXL program
    const int treeShift = reservedTrees - tree_firstUserTree;

    // (i) shift tree pointers in kids (*)
    for (int c = tree_firstUserKid; c <= tree_kidCount; c++) {
        if ((tree_kids[c]) >= tree_firstUserTree) {
            tree_kids[c] += treeShift;
        }
    }

    // (ii) shift tree pointers in rules (*)
    for (int i = 1; i <= rule_nRules; i++) {
        struct ruleT *r = &(rule_rules[i]);

        // rules ->patternTP
        if (r->patternTP >= tree_firstUserTree) {
            r->patternTP += treeShift;
        }

        // rules ->replacementTP
        if (r->replacementTP >= tree_firstUserTree) {
            r->replacementTP += treeShift;
        }

        // rules ->prePattern.partsBase ->patternTP, rules ->prePattern.partsBase ->replacementTP
        for (int p = 1; p <= r->prePattern.nparts; p++) {
            if (rule_ruleParts[r->prePattern.partsBase + p].patternTP >= tree_firstUserTree) {
                rule_ruleParts[r->prePattern.partsBase + p].patternTP += treeShift;
            }
            if (rule_ruleParts[r->prePattern.partsBase + p].replacementTP >= tree_firstUserTree) {
                rule_ruleParts[r->prePattern.partsBase + p].replacementTP += treeShift;
            }
        }

        // rules ->postPattern.partsBase ->patternTP, rules ->postPattern.partsBase ->replacementTP
        for (int p = 1; p <= r->postPattern.nparts; p++) {
            if (rule_ruleParts[r->postPattern.partsBase + p].patternTP >= tree_firstUserTree) {
                rule_ruleParts[r->postPattern.partsBase + p].patternTP += treeShift;
            }
            if (rule_ruleParts[r->postPattern.partsBase + p].replacementTP >= tree_firstUserTree) {
                rule_ruleParts[r->postPattern.partsBase + p].replacementTP += treeShift;
            }
        }
    }

    // (iii) shift inputGrammarTreeTP
    if (inputGrammarTreeTP >= tree_firstUserTree) {
        inputGrammarTreeTP += treeShift;
    }

    // (iv) shift tree pointers in the symbol table
    for (int s = 1; s <= symbol_nSymbols; s++) {
        symbol_symbols[s] += treeShift;
    }

    // (v) move the actual shifted trees down to the beginning of the trees array
    for (int n = tree_firstUserTree; n <= tree_treeCount; n++) {
        tree_trees[n + treeShift] = tree_trees[n];
    }

    // document what we've done
    tree_firstUserTree += treeShift;
    tree_treeCount += treeShift;
}

void loadstore_save (string tofile)
{
    // Make sure that we didn't run out of space compiling 
    // (Otherwise the save/restore cannot work)
    if (tree_allocationStrategy != simple) {
        error ("", "Not enough memory to compile TXL program (a larger size is required for this program)", FATAL, 401);
    }

    // Compress the trees and kids in the TXL program to maximize available transformation space when run
    loadstore_CompressKids ();
    loadstore_CompressTrees ();

    // Now store the compiled TXL program as structured bytecode
    int tf;
    tfopen (OPEN_BINARY_WRITE, tofile, &tf);

    if (tf == 0) {
        string message;
        stringprintf (message, "Unable to create TXL bytecode file '%s'", tofile);
        error ("", message, FATAL, 409);
    }

    // 0. Header
    int nbytes = 0;
    tfwrite (&loadstore_MAGNUM, sizeof loadstore_MAGNUM, tf);
    tfwrite (&options_txlSize, sizeof options_txlSize, tf);     // MUST be first after MAGNUM
    nbytes += 8;

    unsigned char vlen = stringlen (version);
    string vversion;
    stringcpy (vversion, version);
    tfwrite (&vlen, sizeof vlen, tf);
    tfwrite (vversion, (int) (vlen + 1), tf);
    nbytes += vlen + 2;
    vlen = stringlen (options_txlSourceFileName);
    tfwrite (&vlen, sizeof vlen, tf);
    tfwrite (options_txlSourceFileName, (int) (vlen + 1), tf);
    nbytes += vlen + 2;
    tfwrite (&loadstore_MAGNUM, sizeof loadstore_MAGNUM, tf);
    nbytes += 4;

    if (options_option[verbose_p]) {
        fprintf (stderr, "Header: %d\n", nbytes);
    }

    // 1. compoundTokens and commentTokens
    nbytes = 0;
    tfwrite (&scanner_nCompounds, sizeof scanner_nCompounds, tf);
    nbytes += 4;
    for (int i = 1; i <= scanner_nCompounds; i++) {
        unsigned char nc;
        nc = scanner_compoundTokens[i].length;
        tfwrite (&nc, sizeof nc, tf);
        tfwrite (scanner_compoundTokens[i].literal, (nc + 1), tf);
        nbytes += nc + 2;
    }
    for (int i = 0; i <= 255; i++) {
        unsigned char ci = scanner_compoundIndex[i];
        tfwrite (&ci, sizeof ci, tf);
    }
    nbytes += 256;
    tfwrite (&scanner_nComments, sizeof scanner_nComments, tf);
    nbytes += 4;
    tfwrite (&(scanner_commentStart[1]), (int) (scanner_nComments * sizeof (tokenT)), tf);
    nbytes += scanner_nComments * sizeof (tokenT);
    tfwrite (&(scanner_commentEnd[1]), (int) (scanner_nComments * sizeof (tokenT)), tf);
    nbytes += scanner_nComments * sizeof (tokenT);
    tfwrite (&loadstore_MAGNUM, sizeof loadstore_MAGNUM, tf);
    nbytes += 4;

    if (options_option[verbose_p]) {
        fprintf (stderr, "Compounds/comments: %d\n", nbytes);
    }

    // 2. keywords
    tfwrite (&scanner_nKeys, sizeof scanner_nKeys, tf);
    tfwrite (&(scanner_keywordTokens[1]), (int) (scanner_nKeys * sizeof (tokenT)), tf);
    tfwrite (&loadstore_MAGNUM, sizeof loadstore_MAGNUM, tf);

    if (options_option[verbose_p]) {
        fprintf (stderr, "Keywords: %d\n", (int) (8 + (scanner_nKeys * sizeof (tokenT))));
    }

    // 3. tokenPatterns
    nbytes = 0;
    tfwrite (&scanner_nPatterns, sizeof scanner_nPatterns, tf);
    nbytes += 4;
    for (int i = 1; i <= scanner_nPatterns; i++) {
        struct scanner_patternEntryT *tp = &(scanner_tokenPatterns[i]);
        tfwrite (&(tp->kind), sizeof (tp->kind), tf);
        tfwrite (&(tp->name), sizeof (tp->name), tf);
        nbytes += 9;
        short tplen = tp->length;
        tfwrite (&tplen, sizeof tplen, tf);
        for (int j = 1; j <= (tplen + 1); j++) {
            short pj;
            pj = tp->pattern[j];
            tfwrite (&pj, sizeof pj, tf);
        }
        nbytes += 2 + (2 * (tplen + 1));
    }
    for (int i = 0; i <= 255; i++) {
        unsigned char pi;
        pi = scanner_patternIndex[i];
        tfwrite (&pi, sizeof pi, tf);
    }
    nbytes += 256;
    tfwrite (&scanner_patternNLCommentIndex, sizeof scanner_patternNLCommentIndex, tf);
    nbytes += 4;
    tfwrite (&scanner_nPatternLinks, sizeof scanner_nPatternLinks, tf);
    nbytes += 4;
    for (int i = 1; i <= scanner_nPatternLinks; i++) {
        unsigned char pi;
        pi = scanner_patternLink[i];
        tfwrite (&pi, sizeof pi, tf);
    }
    nbytes += scanner_nPatternLinks;
    tfwrite (&(kindType[firstUserTokenKind]), (int) (((lastUserTokenKind - firstUserTokenKind) + 1) * sizeof (tokenT)), tf);
    nbytes += ((lastUserTokenKind - firstUserTokenKind) + 1) * sizeof (tokenT);
    tfwrite (&loadstore_MAGNUM, sizeof loadstore_MAGNUM, tf);
    nbytes += 4;

    if (options_option[verbose_p]) {
        fprintf (stderr, "Patterns: %d\n", nbytes);
    }

    // 4. idents
    nbytes = 0;
    tfwrite (&ident_nIdentChars, sizeof ident_nIdentChars, tf);
    tfwrite (&(ident_identText[0]), ident_nIdentChars, tf);
    nbytes += 4 + ident_nIdentChars;
    tfwrite (&ident_nIdents, sizeof ident_nIdents, tf);
    for (int i = 0; i <= maxIdents - 1; i++) {
        if (ident_idents[i] != nilIdent) {
            int ii = i;
            // encode identifier text address relative identifier text array
            addressint idtai = (addressint) (ident_idents[i]) - (addressint) (&(ident_identText[0]));
            int idti = idtai;
            tfwrite (&ii, sizeof ii, tf);
            tfwrite (&idti, sizeof idti, tf);
            tfwrite (&(ident_identKind[i]), sizeof (ident_identKind[i]), tf);
            nbytes += 9;
        }
    }
    tfwrite (&loadstore_MAGNUM, sizeof loadstore_MAGNUM, tf);

    if (options_option[verbose_p]) {
        fprintf (stderr, "IdentTable: %d\n", (4 + nbytes));
    }

    // 5. trees
    tfwrite (&tree_treeCount, sizeof tree_treeCount, tf);
    tfwrite (&tree_firstUserTree, sizeof tree_firstUserTree, tf);
    tfwrite (&(tree_trees[tree_firstUserTree]), (((tree_treeCount - tree_firstUserTree) + 1) * sizeof (struct parseTreeT)), tf);
    tfwrite (&loadstore_MAGNUM, sizeof loadstore_MAGNUM, tf);

    if (options_option[verbose_p]) {
        fprintf (stderr, "Trees: %d\n", (int) (12 + (((tree_treeCount - tree_firstUserTree) + 1) * sizeof (struct parseTreeT))));
    }

    // 6. kids
    tfwrite (&tree_kidCount, sizeof tree_kidCount, tf);
    tfwrite (&tree_firstUserKid, sizeof tree_firstUserKid, tf);
    tfwrite (&(tree_kids[tree_firstUserKid]), (((tree_kidCount - tree_firstUserKid) + 1) * sizeof (treePT)), tf);
    tfwrite (&inputGrammarTreeTP, sizeof inputGrammarTreeTP, tf);
    tfwrite (&loadstore_MAGNUM, sizeof loadstore_MAGNUM, tf);

    if (options_option[verbose_p]) {
        fprintf (stderr, "Kids: %d\n", (int) (12 + (((tree_kidCount - tree_firstUserKid) + 1) * sizeof (treePT))));
    }

    // 7. symbols
    tfwrite (&symbol_nSymbols, sizeof symbol_nSymbols, tf);
    tfwrite (&(symbol_symbols[1]), (symbol_nSymbols * sizeof (treePT)), tf);
    tfwrite (&loadstore_MAGNUM, sizeof loadstore_MAGNUM, tf);

    if (options_option[verbose_p]) {
        fprintf (stderr, "SymbolTable: %d\n", (int) (4 + (symbol_nSymbols * sizeof (treePT))));
    }

    // 8. rules
    tfwrite (&rule_nRules, sizeof rule_nRules, tf);
    nbytes = 0;
    for (int i = 1; i <= rule_nRules; i++) {
        struct ruleT *r = (&(rule_rules[i]));
        tfwrite (&(r->name), sizeof (r->name), tf);
        tfwrite (&(r->target), sizeof (r->target), tf);
        tfwrite (&(r->skipName), sizeof (r->skipName), tf);
        tfwrite (&(r->skipName2), sizeof (r->skipName2), tf);
        tfwrite (&(r->skipName3), sizeof (r->skipName3), tf);
        tfwrite (&(r->patternTP), sizeof (r->patternTP), tf);
        tfwrite (&(r->replacementTP), sizeof (r->replacementTP), tf);
        tfwrite (&(r->kind), sizeof (r->kind), tf);
        tfwrite (&(r->starred), sizeof (r->starred), tf);
        tfwrite (&(r->isCondition), sizeof (r->isCondition), tf);
        tfwrite (&(r->defined), sizeof (r->defined), tf);
        tfwrite (&(r->called), sizeof (r->called), tf);
        tfwrite (&(r->skipRepeat), sizeof (r->skipRepeat), tf);
        tfwrite (&(r->prePattern.nparts), sizeof (r->prePattern.nparts), tf);
        for (int pp = 1; pp <= r->prePattern.nparts; pp++) {
            tfwrite (&(rule_ruleParts[r->prePattern.partsBase + pp]), sizeof (struct rulePartT), tf);
        }
        tfwrite (&(r->postPattern.nparts), sizeof (r->postPattern.nparts), tf);
        for (int pp = 1; pp <= r->postPattern.nparts; pp++) {
            tfwrite (&(rule_ruleParts[r->postPattern.partsBase + pp]), sizeof (struct rulePartT), tf);
        }
        tfwrite (&(r->localVars.nformals), sizeof (r->localVars.nformals), tf);
        tfwrite (&(r->localVars.nprelocals), sizeof (r->localVars.nprelocals), tf);
        tfwrite (&(r->localVars.nlocals), sizeof (r->localVars.nlocals), tf);
        for (int lc = 1; lc <= r->localVars.nlocals; lc++) {
            tfwrite (&(rule_ruleLocals[r->localVars.localsBase + lc]), sizeof (struct ruleLocalT), tf);
        }
        nbytes += (60 + (((r->prePattern.nparts) + (r->postPattern.nparts)) * sizeof (struct rulePartT))) 
            + ((r->localVars.nlocals) * sizeof (struct ruleLocalT));
    }
    tfwrite (&mainRule, sizeof mainRule, tf);
    tfwrite (&loadstore_MAGNUM, sizeof loadstore_MAGNUM, tf);

    if (options_option[verbose_p]) {
        fprintf (stderr, "Rules: %d\n", (12 + nbytes));
    }

    // 9. options
    options_option[compile_p] = false;
    options_option[load_p] = true;
    tfwrite (options_option, sizeof options_option, tf);
    options_option[compile_p] = true;
    options_option[load_p] = false;
    nbytes = sizeof (options_option);
    unsigned char olen = stringlen (options_txlLibrary);
    tfwrite (&olen, sizeof olen, tf);
    tfwrite (options_txlLibrary, (olen + 1), tf);
    nbytes += olen + 2;
    tfwrite (&options_outputLineLength, sizeof options_outputLineLength, tf);
    nbytes += 4;
    tfwrite (&options_indentIncrement, sizeof options_indentIncrement, tf);
    nbytes += 4;
    olen = stringlen (options_optionIdChars);
    tfwrite (&olen, sizeof olen, tf);
    tfwrite (options_optionIdChars, (olen + 1), tf);
    nbytes += olen + 2;
    olen = stringlen (options_optionSpChars);
    tfwrite (&olen, sizeof olen, tf);
    tfwrite (options_optionSpChars, (olen + 1), tf);
    nbytes += olen + 2;
    tfwrite (&charset_stringlitEscapeChar, sizeof charset_stringlitEscapeChar, tf);
    tfwrite (&charset_charlitEscapeChar, sizeof charset_charlitEscapeChar, tf);
    nbytes += 2;
    tfwrite (&loadstore_MAGNUM, sizeof loadstore_MAGNUM, tf);
    nbytes += 4;

    if (options_option[verbose_p]) {
        fprintf (stderr, "Options: %d\n", nbytes);
    }

    // 10. EOF
    tfwrite (&loadstore_MAGNUM, sizeof loadstore_MAGNUM, tf);

    if (options_option[verbose_p]) {
        fprintf (stderr, "Trailer: %d\n", 4);
    }

    tfclose (tf);
}

static void loadstore_synchError (int n)
{
    string message;
    stringprintf (message, "Synchronization error %d on compiled file", n);
    error ("", message, INTERNAL_FATAL, 405);
}

#ifndef STANDALONE
// Stored bytecode (.ctxl) version, for use with -c/-l command line arguments

void loadstore_restore (string fromfile)
{
    // Read the compiled TXL program as structured bytecode
    int tf;
    tfopen (OPEN_BINARY_READ, fromfile, &tf);

    // Header
    if (options_option[verbose_p]) {
        fprintf (stderr, "Header\n");
    }

    int mn;
    tfread (&mn, sizeof mn, tf);
    if (mn != loadstore_MAGNUM) {
        error ("", "Load file is not a compiled TXL object file", FATAL, 402);
    }

    tfread (&mn, sizeof mn, tf);
    if (mn != options_txlSize) {
        error ("", "TXL size does not match compiled size", INTERNAL_FATAL, 403);
    }

    unsigned char vlen;
    tfread (&vlen, sizeof vlen, tf);
    string storedversion;
    tfread (storedversion, (vlen + 1), tf);

    if (stringcmp (storedversion, version) != 0) {
        error ("", "TXL object file was compiled by a different version of TXL", FATAL, 404);
    }

    tfread (&vlen, sizeof vlen, tf);
    tfread (&options_txlSourceFileName, (vlen + 1), tf);

    tfread (&mn, sizeof mn, tf);
    if (mn != loadstore_MAGNUM) {
        loadstore_synchError (0);
    }

    // 1. compoundTokens and commentTokens
    if (options_option[verbose_p]) {
        fprintf (stderr, "Compounds/comments\n");
    }

    tfread (&scanner_nCompounds, sizeof (scanner_nCompounds), tf);
    for (int i = 1; i <= scanner_nCompounds; i++) {
        unsigned char nc;
        tfread (&nc, sizeof nc, tf);
        scanner_compoundTokens[i].length = nc;
        tfread (&(scanner_compoundTokens[i].literal), (nc + 1), tf);
    }
    for (int i = 0; i <= 255; i++) {
        unsigned char ci;
        tfread (&ci, sizeof ci, tf);
        scanner_compoundIndex[i] = ci;
    }
    tfread (&scanner_nComments, sizeof (scanner_nComments), tf);
    tfread (&(scanner_commentStart[1]), (scanner_nComments * sizeof (tokenT)), tf);
    tfread (&(scanner_commentEnd[1]), (scanner_nComments * sizeof (tokenT)), tf);

    tfread (&mn, sizeof mn, tf);
    if (mn != loadstore_MAGNUM) {
        loadstore_synchError (1);
    }

    // 2. keywords
    if (options_option[verbose_p]) {
        fprintf (stderr, "Keywords\n");
    }

    tfread (&scanner_nKeys, sizeof (scanner_nKeys), tf);
    tfread (&(scanner_keywordTokens[1]), (scanner_nKeys * sizeof (tokenT)), tf);

    tfread (&mn, sizeof mn, tf);
    if (mn != loadstore_MAGNUM) {
        loadstore_synchError (2);
    }

    // tokenPatterns
    if (options_option[verbose_p]) {
        fprintf (stderr, "Patterns\n");
    }

    tfread (&scanner_nPatterns, sizeof (scanner_nPatterns), tf);
    for (int i = 1; i <= scanner_nPatterns; i++) {
        struct scanner_patternEntryT *tp = &(scanner_tokenPatterns[i]);
        tfread (&(tp->kind), sizeof (tp->kind), tf);
        tfread (&(tp->name), sizeof (tp->name), tf);
        short slen;
        tfread (&slen, sizeof slen, tf);
        tp->length = slen;
        for (int j = 1; j <= slen + 1; j++) {
            short pj;
            tfread (&pj, sizeof pj, tf);
            tp->pattern[j] = pj;
        }
    }

    for (int i = 0; i <= 255; i++) {
        unsigned char pi;
        tfread (&pi, sizeof pi, tf);
        scanner_patternIndex[i] = pi;
    }

    tfread (&scanner_patternNLCommentIndex, sizeof (scanner_patternNLCommentIndex), tf);
    tfread (&scanner_nPatternLinks, sizeof (scanner_nPatternLinks), tf);
    for (int i = 1; i <= scanner_nPatternLinks; i++) {
        unsigned char pi;
        tfread (&pi, sizeof pi, tf);
        scanner_patternLink[i] = pi;
    }

    tfread (&(kindType[firstUserTokenKind]), ((lastUserTokenKind - firstUserTokenKind) + 1) * sizeof (tokenT), tf);

    tfread (&mn, sizeof mn, tf);
    if (mn != loadstore_MAGNUM) {
        loadstore_synchError (3);
    }

    // 4. idents
    if (options_option[verbose_p]) {
        fprintf (stderr, "IdentTable\n");
    }

    tfread (&ident_nIdentChars, sizeof (ident_nIdentChars), tf);
    tfread (&(ident_identText[0]), ident_nIdentChars, tf);
    tfread (&ident_nIdents, sizeof (ident_nIdents), tf);
    while (true) {
        int i;
        tfread (&i, sizeof i, tf);

        if (i == loadstore_MAGNUM) break;

        int idti;
        tfread (&idti, sizeof idti, tf);
        ident_idents[i] = (longstring *) ((&(ident_identText[0])) + idti);
        tfread (&(ident_identKind[i]), sizeof (ident_identKind[i]), tf);
    }

    if (mn != loadstore_MAGNUM) {
        loadstore_synchError (4);
    }

    // 5. Trees
    if (options_option[verbose_p]) {
        fprintf (stderr, "Trees\n");
    }

    tfread (&tree_treeCount, sizeof (tree_treeCount), tf);
    tfread (&tree_firstUserTree, sizeof (tree_firstUserTree), tf);
    tfread (&(tree_trees[tree_firstUserTree]), ((tree_treeCount - tree_firstUserTree) + 1) * sizeof (struct parseTreeT), tf);

    tfread (&mn, sizeof mn, tf);
    if (mn != loadstore_MAGNUM) {
        loadstore_synchError (5);
    }

    // 6. kids
    if (options_option[verbose_p]) {
        fprintf (stderr, "Kids\n");
    }

    tfread (&tree_kidCount, sizeof (tree_kidCount), tf);
    tfread (&tree_firstUserKid, sizeof (tree_firstUserKid), tf);
    tfread (&(tree_kids[tree_firstUserKid]), ((tree_kidCount - tree_firstUserKid) + 1) * sizeof (treePT), tf);
    tfread (&inputGrammarTreeTP, sizeof inputGrammarTreeTP, tf);

    tfread (&mn, sizeof mn, tf);
    if (mn != loadstore_MAGNUM) {
        loadstore_synchError (6);
    }

    // 7. Symbols
    if (options_option[verbose_p]) {
        fprintf (stderr, "SymbolTable\n");
    }

    tfread (&symbol_nSymbols, sizeof (symbol_nSymbols), tf);
    tfread (&(symbol_symbols[1]), symbol_nSymbols * sizeof (treePT), tf);

    tfread (&mn, sizeof mn, tf);
    if (mn != loadstore_MAGNUM) {
        loadstore_synchError (7);
    }

    // 8. Rules
    if (options_option[verbose_p]) {
        fprintf (stderr, "Rules\n");
    }

    tfread (&rule_nRules, sizeof (rule_nRules), tf);
    for (int i = 1; i <= rule_nRules; i++) {
        struct ruleT *r = &(rule_rules[i]);
        tfread (&(r->name), sizeof (r->name), tf);
        tfread (&(r->target), sizeof (r->target), tf);
        tfread (&(r->skipName), sizeof (r->skipName), tf);
        tfread (&(r->skipName2), sizeof (r->skipName2), tf);
        tfread (&(r->skipName3), sizeof (r->skipName3), tf);
        tfread (&(r->patternTP), sizeof (r->patternTP), tf);
        tfread (&(r->replacementTP), sizeof (r->replacementTP), tf);
        tfread (&(r->kind), sizeof (r->kind), tf);
        tfread (&(r->starred), sizeof (r->starred), tf);
        tfread (&(r->isCondition), sizeof (r->isCondition), tf);
        tfread (&(r->defined), sizeof (r->defined), tf);
        tfread (&(r->called), sizeof (r->called), tf);
        tfread (&(r->skipRepeat), sizeof (r->skipRepeat), tf);
        r->prePattern.partsBase = rule_rulePartCount;
        tfread (&(r->prePattern.nparts), sizeof (r->prePattern.nparts), tf);
        for (int pp = 1; pp <= r->prePattern.nparts; pp++) {
            tfread (&(rule_ruleParts[r->prePattern.partsBase + pp]), sizeof (struct rulePartT), tf);
        }
        rule_rulePartCount += r->prePattern.nparts;
        r->postPattern.partsBase = rule_rulePartCount;
        tfread (&(r->postPattern.nparts), sizeof (r->postPattern.nparts), tf);
        for (int pp = 1; pp <= r->postPattern.nparts; pp++) {
            tfread (&(rule_ruleParts[r->postPattern.partsBase + pp]), sizeof (struct rulePartT), tf);
        }
        rule_rulePartCount += r->postPattern.nparts;
        r->localVars.localsBase = rule_ruleLocalCount;
        tfread (&(r->localVars.nformals), sizeof (r->localVars.nformals), tf);
        tfread (&(r->localVars.nprelocals), sizeof (r->localVars.nprelocals), tf);
        tfread (&(r->localVars.nlocals), sizeof (r->localVars.nlocals), tf);
        for (int lc = 1; lc <= r->localVars.nlocals; lc++) {
            tfread (&(rule_ruleLocals[r->localVars.localsBase + lc]), sizeof (struct ruleLocalT), tf);
        }
        rule_ruleLocalCount += r->localVars.nlocals;
        r->calledRules.callsBase = 0;
        r->calledRules.ncalls = 0;
    }
    tfread (&mainRule, sizeof mainRule, tf);

    tfread (&mn, sizeof mn, tf);
    if (mn != loadstore_MAGNUM) {
        loadstore_synchError (8);
    }

    // 9. Options
    if (options_option[verbose_p]) {
        fprintf (stderr, "Options\n");
    }

    unsigned char olen;
    tfread (&options_option, sizeof (options_option), tf);
    tfread (&olen, sizeof olen, tf);
    tfread (options_txlLibrary, (olen + 1), tf);
    tfread (&options_outputLineLength, sizeof (options_outputLineLength), tf);
    tfread (&options_indentIncrement, sizeof (options_indentIncrement), tf);
    tfread (&olen, sizeof olen, tf);
    tfread (options_optionIdChars, (olen + 1), tf);
    for (int i = 1; i <= olen; i++) {
        unsigned char idc;
        idc = options_optionIdChars[(i - 1)];
        charset_addIdChar (idc, true);
    }
    tfread (&olen, sizeof olen, tf);
    tfread (options_optionSpChars, (olen + 1), tf);
    for (int i = 1; i <= olen; i++) {
        unsigned char spc;
        spc = options_optionSpChars[(i - 1)];
        charset_addSpaceChar (spc, true);
    }

    unsigned char c;
    tfread (&c, sizeof c, tf);
    charset_setEscapeChar (c, true);
    tfread (&c, sizeof c, tf);
    charset_setEscapeChar (c, true);

    tfread (&mn, sizeof mn, tf);
    if (mn != loadstore_MAGNUM) {
        loadstore_synchError (9);
    }

    // 10. EOF
    if (options_option[verbose_p]) {
        fprintf (stderr, "EOF\n");
    }

    tfread (&mn, sizeof mn, tf);
    if (mn != loadstore_MAGNUM) {
        loadstore_synchError (10);
    }

    tfclose (tf);
}

#else // STANDALONE

// Standalone TXL app version - bytecode is stored in C program's 'TXL_CTXL' byte array 

// Bytecode array access routines 
extern addressint TXL_CTXL;
typedef unsigned char loadstore_bytearray[1000000000];
static int loadstore_ctxlptr;  // 0

static void loadstore_ctxlinitialize (void) {
    loadstore_ctxlptr = 0;
}

static void loadstore_ctxlgetint (addressint target)
{
    for (int b = 0; b < sizeof (int); b++) {
        (* (loadstore_bytearray *) target)[b] = (* (loadstore_bytearray *) TXL_CTXL)[loadstore_ctxlptr];
        loadstore_ctxlptr += 1;
    }
}

static void loadstore_ctxlgetnat2 (addressint target)
{
    for (int b = 0; b < sizeof (unsigned short); b++) {
        (* (loadstore_bytearray *) target)[b] = (* (loadstore_bytearray *) TXL_CTXL)[loadstore_ctxlptr];
        loadstore_ctxlptr += 1;
    }
}

static void loadstore_ctxlgetboolean (addressint target)
{
    for (int b = 0; b < sizeof (bool); b++) {
        (* (loadstore_bytearray *) target)[b] = (* (loadstore_bytearray *) TXL_CTXL)[loadstore_ctxlptr];
        loadstore_ctxlptr += 1;
    }
}

static void loadstore_ctxlgetbyte (addressint target)
{
    (* (unsigned char *) target) = (* (loadstore_bytearray *) TXL_CTXL)[loadstore_ctxlptr];
    loadstore_ctxlptr += 1;
}

static void loadstore_ctxlgetbytes (addressint target, int nbytes)
{
    for (int b = 0; b < nbytes; b++) {
        (* (loadstore_bytearray *) target)[b] = (* (loadstore_bytearray *) TXL_CTXL)[loadstore_ctxlptr];
        loadstore_ctxlptr += 1;
    }
}

void loadstore_restore (string fromfile)
{
    loadstore_ctxlinitialize ();

    // 0. Header
    int mn;
    loadstore_ctxlgetint ((addressint) (&(mn)));
    if (mn != loadstore_MAGNUM) {
        error ("", "Load file is not a compiled TXL object file", FATAL, 402);
    }

    loadstore_ctxlgetint ((addressint) (&(mn)));
    if (mn != options_txlSize) {
        error ("", "TXL size does not match compiled size", INTERNAL_FATAL, 403);
    }

    unsigned char vlen;
    string storedversion;
    loadstore_ctxlgetbyte ((addressint) (&(vlen)));
    loadstore_ctxlgetbytes ((addressint) (storedversion), (vlen + 1));
    if (stringcmp (storedversion, version) != 0) {
        error ("", "TXL object file was compiled by a different version of TXL", FATAL, 404);
    }

    loadstore_ctxlgetbyte ((addressint) (&(vlen)));
    loadstore_ctxlgetbytes ((addressint) (options_txlSourceFileName), (vlen + 1));

    loadstore_ctxlgetint ((addressint) (&(mn)));
    if (mn != loadstore_MAGNUM) {
        loadstore_synchError (0);
    }

    // 1. compoundTokens and commentTokens
    loadstore_ctxlgetint ((addressint) (&(scanner_nCompounds)));
    for (int i = 1; i <= scanner_nCompounds; i++) {
        unsigned char len;
        loadstore_ctxlgetbyte ((addressint) (&(len)));
        scanner_compoundTokens[i].length = len;
        loadstore_ctxlgetbytes ((addressint) (scanner_compoundTokens[i].literal), (len + 1));
    }

    for (int i = 0; i <= 255; i++) {
        unsigned char ci;
        loadstore_ctxlgetbyte ((addressint) (&(ci)));
        scanner_compoundIndex[i] = ci;
    }

    loadstore_ctxlgetint ((addressint) (&(scanner_nComments)));
    loadstore_ctxlgetbytes ((addressint) (&(scanner_commentStart[1])), (scanner_nComments * sizeof (tokenT)));
    loadstore_ctxlgetbytes ((addressint) (&(scanner_commentEnd[1])), (scanner_nComments * sizeof (tokenT)));

    loadstore_ctxlgetint ((addressint) (&(mn)));
    if (mn != loadstore_MAGNUM) {
        loadstore_synchError (1);
    }

    // 2. keywords
    loadstore_ctxlgetint ((addressint) (&(scanner_nKeys)));
    loadstore_ctxlgetbytes ((addressint) (&(scanner_keywordTokens[1])), (scanner_nKeys * sizeof (tokenT)));

    loadstore_ctxlgetint ((addressint) (&(mn)));
    if (mn != loadstore_MAGNUM) {
        loadstore_synchError (2);
    }

    // 3. tokenPatterns
    loadstore_ctxlgetint ((addressint) (&(scanner_nPatterns)));
    for (int i = 1; i <= scanner_nPatterns; i++) {
        short len;
        struct scanner_patternEntryT *tp = &(scanner_tokenPatterns[i]);
        loadstore_ctxlgetbytes ((addressint) (&(tp->kind)), sizeof (tp->kind));
        loadstore_ctxlgetbytes ((addressint) (&(tp->name)), sizeof (tp->name));
        loadstore_ctxlgetbytes ((addressint) (&(len)), sizeof (len));
        tp->length = len;
        for (int j = 1; j <= (len + 1); j++) {
            short pj;
            loadstore_ctxlgetbytes ((addressint) (&(pj)), sizeof (pj));
            tp->pattern[j] = pj;
        }
    }

    for (int i = 0; i <= 255; i++) {
        unsigned char pi;
        loadstore_ctxlgetbyte ((addressint) (&(pi)));
        scanner_patternIndex[i] = pi;
    }

    loadstore_ctxlgetint ((addressint) (&(scanner_patternNLCommentIndex)));
    loadstore_ctxlgetint ((addressint) (&(scanner_nPatternLinks)));
    for (int i = 1; i <= scanner_nPatternLinks; i++) {
        unsigned char pi;
        loadstore_ctxlgetbyte ((addressint) (&(pi)));
        scanner_patternLink[i] = pi;
    }

    loadstore_ctxlgetbytes ((addressint) (&(kindType[firstUserTokenKind])), ((lastUserTokenKind - firstUserTokenKind) + 1) * sizeof (tokenT));

    loadstore_ctxlgetint ((addressint) (&(mn)));
    if (mn != loadstore_MAGNUM) {
        loadstore_synchError (3);
    }

    // 4. Idents
    loadstore_ctxlgetint ((addressint) (&(ident_nIdentChars)));
    loadstore_ctxlgetbytes ((addressint) (&(ident_identText[0])), ident_nIdentChars);
    loadstore_ctxlgetint ((addressint) (&(ident_nIdents)));
    while (true) {
        int i;
        loadstore_ctxlgetint ((addressint) (&(i)));

        if (i == loadstore_MAGNUM) break;

        int idti;
        loadstore_ctxlgetint ((addressint) (&(idti)));
        ident_idents[i] = (longstring *) ((&(ident_identText[0])) + idti);
        loadstore_ctxlgetbytes ((addressint) (&(ident_identKind[i])), sizeof (enum treeKindT));
    }

    if (mn != loadstore_MAGNUM) {
        loadstore_synchError (4);
    }

    // 5. trees
    loadstore_ctxlgetint ((addressint) (&(tree_treeCount)));
    loadstore_ctxlgetint ((addressint) (&(tree_firstUserTree)));
    loadstore_ctxlgetbytes ((addressint) (&(tree_trees[tree_firstUserTree])), ((tree_treeCount - tree_firstUserTree) + 1) * sizeof (struct parseTreeT));

    loadstore_ctxlgetint ((addressint) (&(mn)));
    if (mn != loadstore_MAGNUM) {
        loadstore_synchError (5);
    }

    // 6. kids
    loadstore_ctxlgetint ((addressint) (&(tree_kidCount)));
    loadstore_ctxlgetint ((addressint) (&(tree_firstUserKid)));
    loadstore_ctxlgetbytes ((addressint) (&(tree_kids[tree_firstUserKid])), ((tree_kidCount - tree_firstUserKid) + 1) * sizeof (treePT));
    loadstore_ctxlgetint ((addressint) (&(inputGrammarTreeTP)));

    loadstore_ctxlgetint ((addressint) (&(mn)));
    if (mn != loadstore_MAGNUM) {
        loadstore_synchError (6);
    }

    // 7. Symbols
    loadstore_ctxlgetint ((addressint) (&(symbol_nSymbols)));
    loadstore_ctxlgetbytes ((addressint) (&(symbol_symbols[1])), symbol_nSymbols * sizeof (treePT));

    loadstore_ctxlgetint ((addressint) (&(mn)));
    if (mn != loadstore_MAGNUM) {
        loadstore_synchError (7);
    }

    // 8. Rules
    loadstore_ctxlgetint ((addressint) (&(rule_nRules)));
    for (int i = 1; i <= rule_nRules; i++) {
        struct ruleT *r = &(rule_rules[i]);
        loadstore_ctxlgetbytes ((addressint) (&(r->name)), sizeof (r->name));
        loadstore_ctxlgetbytes ((addressint) (&(r->target)), sizeof (r->target));
        loadstore_ctxlgetbytes ((addressint) (&(r->skipName)), sizeof (r->skipName));
        loadstore_ctxlgetbytes ((addressint) (&(r->skipName2)), sizeof (r->skipName2));
        loadstore_ctxlgetbytes ((addressint) (&(r->skipName3)), sizeof (r->skipName3));
        loadstore_ctxlgetint ((addressint) (&(r->patternTP)));
        loadstore_ctxlgetint ((addressint) (&(r->replacementTP)));
        loadstore_ctxlgetbytes ((addressint) (&(r->kind)), sizeof (r->kind));
        loadstore_ctxlgetboolean ((addressint) (&(r->starred)));
        loadstore_ctxlgetboolean ((addressint) (&(r->isCondition)));
        loadstore_ctxlgetboolean ((addressint) (&(r->defined)));
        loadstore_ctxlgetboolean ((addressint) (&(r->called)));
        loadstore_ctxlgetboolean ((addressint) (&(r->skipRepeat)));

        r->prePattern.partsBase = rule_rulePartCount;
        loadstore_ctxlgetnat2((addressint) (&(r->prePattern.nparts)));
        for (int pp = 1; pp <= r->prePattern.nparts; pp++) {
            loadstore_ctxlgetbytes ((addressint) (&(rule_ruleParts[r->prePattern.partsBase + pp])), sizeof (struct rulePartT));
        }
        rule_rulePartCount += r->prePattern.nparts;

        r->postPattern.partsBase = rule_rulePartCount;
        loadstore_ctxlgetnat2((addressint) (&(r->postPattern.nparts)));
        for (int pp = 1; pp <= r->postPattern.nparts; pp++) {
            loadstore_ctxlgetbytes ((addressint) (&(rule_ruleParts[r->postPattern.partsBase + pp])), sizeof (struct rulePartT));
        }
        rule_rulePartCount += r->postPattern.nparts;

        r->localVars.localsBase = rule_ruleLocalCount;
        loadstore_ctxlgetnat2((addressint) (&(r->localVars.nformals)));
        loadstore_ctxlgetnat2((addressint) (&(r->localVars.nprelocals)));
        loadstore_ctxlgetnat2((addressint) (&(r->localVars.nlocals)));
        for (int lc = 1; lc <= r->localVars.nlocals; lc++) {
            loadstore_ctxlgetbytes ((addressint) (&(rule_ruleLocals[r->localVars.localsBase + lc])), sizeof (struct ruleLocalT));
        }
        rule_ruleLocalCount += r->localVars.nlocals;
        r->calledRules.callsBase = 0;
        r->calledRules.ncalls = 0;
    }

    loadstore_ctxlgetint ((addressint) (&(mainRule)));
    loadstore_ctxlgetint ((addressint) (&(mn)));

    if (mn != loadstore_MAGNUM) {
        loadstore_synchError (8);
    }

    // 9. options
    bool oldoptions[lastOption + 1];
    structassign (oldoptions, options_option);
    loadstore_ctxlgetbytes ((addressint) options_option, sizeof (options_option));
    for (int i = firstOption; i <= lastOption; i++) {
        options_option[i] = ((options_option[i]) || (oldoptions[i]));
    }
    unsigned char len;
    loadstore_ctxlgetbyte ((addressint) (&(len)));
    loadstore_ctxlgetbytes ((addressint) options_txlLibrary, (len + 1));
    int oldOutputLineLength = options_outputLineLength;
    loadstore_ctxlgetint ((addressint) (&(options_outputLineLength)));
    if (oldoptions[width_p]) {
        options_outputLineLength = oldOutputLineLength;
    }
    int oldIndentIncrement = options_indentIncrement;
    loadstore_ctxlgetint ((addressint) (&(options_indentIncrement)));
    if (oldoptions[indent_p]) {
        options_indentIncrement = oldIndentIncrement;
    }
    loadstore_ctxlgetbyte ((addressint) (&(len)));
    loadstore_ctxlgetbytes ((addressint) options_optionIdChars, (len + 1));
    for (int i = 1; i <= len; i++) {
        unsigned char c;
        c = options_optionIdChars[(i - 1)];
        charset_addIdChar (c, true);
    }
    loadstore_ctxlgetbyte ((addressint) (&(len)));
    loadstore_ctxlgetbytes ((addressint) options_optionSpChars, (len + 1));
    for (int i = 1; i <= len; i++) {
        unsigned char c;
        c = options_optionSpChars[(i - 1)];
        charset_addSpaceChar (c, true);
    }
    unsigned char c;
    loadstore_ctxlgetbyte ((addressint) (&(c)));
    charset_setEscapeChar (c, true);
    loadstore_ctxlgetbyte ((addressint) (&(c)));
    charset_setEscapeChar (c, true);
    loadstore_ctxlgetint ((addressint) (&(mn)));

    if (mn != loadstore_MAGNUM) {
        loadstore_synchError (9);
    }

    // 10. EOF
    loadstore_ctxlgetint ((addressint) (&(mn)));
    if (mn != loadstore_MAGNUM) {
        loadstore_synchError (10);
    }
}
#endif

void loadstore (void) {
    loadstore_MAGNUM = 721696969;  // pizza in Karlsruhe
#ifdef STANDALONE
    loadstore_ctxlptr = 0;
#endif
}

